package com.gylang.crawler.crawler;

import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSON;
import com.gylang.crawler.domain.CelebritySimpleInfo;
import com.gylang.crawler.domain.MovieInfo;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.common.xcontent.XContentType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.io.IOException;
import java.util.List;
import java.util.regex.Pattern;

/**
 * @author gylang
 * data 2021/5/26
 */
@Slf4j
@Component
public class MoviePageProcessor implements PageProcessor {
    private final static Pattern celebrity = Pattern.compile("/\\w*/([0-9]*)/");
    private static final Pattern BASE_URL = Pattern.compile("https://movie.douban.com/subject/(\\w*)/");

    @Autowired
    private ElasticsearchRestTemplate restTemplate;

    private static final Site site = Site.me()
//            .addCookie("ll", "%22118281%22")
//            .addCookie("bid", "VgngsTJVEJA")
            .addHeader("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
            .addHeader("Cache-Control", "max-age=0")
            .addHeader("Connection", "keep-alive")
            .addHeader("accept", "*/*")
            .addHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
            .addHeader("content-type", "application/x-www-form-urlencoded; charset=UTF-8")
            .addHeader("x-client-id", "86bb675c13444edf7a083946d2c4e5d9")
            .addHeader("x-request-id", "9f6abb65ccedd84e4448f0f18a20d812")
            .addHeader("x-requested-with", "XMLHttpRequest")
            .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 Edg/90.0.818.66")
            .addHeader("Upgrade-Insecure-Requests", "1")
            .addHeader("Sec-Fetch-User", "?1")
            .addHeader("sec-fetch-site", "same-origin")
            .addHeader("Sec-Fetch-Mode", "navigate")
            .addHeader("Sec-Fetch-Dest", "document")
            .addHeader("sec-ch-ua-mobile", "?0")
            .addHeader("sec-ch-ua", "Not A;Brand\";v=\"99\", \"Chromium\";v=\"90\", \"Microsoft Edge\";v=\"90\"")
            //            .addHeader("Host", "movie.douban.com")
            .setCharset("utf-8")
            .setSleepTime(1000)
            .setTimeOut(20000);

    @Override
    public void process(Page page) {

        Selectable xpath = page.getHtml().xpath("/html/head/script[@type='application/ld+json']");
        if (xpath.match()) {
            // 直接就持久化

            try {
                String json = page.getHtml().xpath("/html/head/script[@type='application/ld+json']").toString();
                json = json
                        .replaceAll("<script type=\"application/ld\\+json\">", "")
                        .replaceAll("</script>", "")
                        .replaceAll("@(\\W*)", "$1");

                MovieInfo movieInfo = JSON.parseObject(json, MovieInfo.class);

                List<CelebritySimpleInfo> celebritySimpleInfoList = CollUtil.newArrayList(movieInfo.getDirector());
                celebritySimpleInfoList.addAll(movieInfo.getActor());
                celebritySimpleInfoList.addAll(movieInfo.getActor());
                celebritySimpleInfoList.stream().filter(c -> StrUtil.isNotBlank(c.getId()))
                        .forEach(c -> c.setId(celebrity.matcher(c.getId()).replaceAll("$1")));
                // 获取
                IndexRequest indexRequest = new IndexRequest("movie_info", "doc", BASE_URL.matcher(page.getUrl().toString()).replaceAll("$1"));
                indexRequest.source(JSON.toJSONString(movieInfo), XContentType.JSON);

                restTemplate.getClient().index(indexRequest, RequestOptions.DEFAULT);
            } catch (Exception e) {
                e.printStackTrace();
                log.error("保存es失败{}  :  {}", page.getUrl(), e.getMessage());
            }
        } else {
            log.error("爬虫失败 {}", page.getUrl().toString());
        }
    }

    @Override
    public Site getSite() {
        return site;
    }
}
